
version = '0.2.0'
author  = 'Mike Taylor'
license = """
Copyright (c) 2007 Mike Taylor
All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

   http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
"""

"""
tf_poll.py

Poll the Twitter Public Timeline using python-twitter and store them.

Requires:
    Python          2.5 is what I'm using - haven't tried it on any other
                    http://python.org

    python-twitter  DeWitt Clinton
                    http://code.google.com/p/python-twitter/

    simplejson      http://undefined.org/python/#simplejson

Contact Info:
    bear            aka Mike Taylor
                    bear42@gmail.com (email and jabber id)
"""

import sys, os
import codecs
import string
import re
import time
import signal
import select
import traceback
import urllib2
import simplejson

from optparse import OptionParser

import twitter
import tf_store


def dumpException(msg):
    """
    Take current traceback info and make it user-friendly
    """
    t, v, tb = sys.exc_info()

    s  = '+------ error ------+\n'
    s += '%s\n' % time.strftime('%H:%M on %A, %d %B')
    s += '=-------------------=\n'
    s += '%s\n' % msg
    s += '=-------------------=\n'
    s += string.join(traceback.format_exception(t, v, tb), '')
    s += '+-------------------+\n'

    sys.stderr.write(s)


mths      = [ 'jan', 'feb', 'mar', 'apr', 'may', 'jun', 'jul', 'aug', 'sep', 'oct', 'nov', 'dec' ]
user_keys = [ 'id', 'name', 'screen_name', 'location', 'description', 'profile_image_url', 'url', 'status' ]

def pollPublicTwitterTimeline():
    """
    Poll the Twitter Public Timeline
    
    Calls the python-twitter GetPublicTimeline() routine, loops thru
    all of the returned status items and then stores them.
    
    The routine maintains a list of Twitter ID's that have already been
    processed as the Twitter folks *love* passing the same information
    regardless of etag values.
    """
    global mths

    def itemTimestamp(date):
        """
        Helper routine to translate the Twitter created_at date string
        to something sortable - used as a key for the queue to make sure
        they are sent out in timestamp order.
        
        Takes "Wed Mar 07 09:32:51 +0000 2007" and returns "2007 03 07 09:32:51 +0000""
        """
        l = date.lower().split(' ')
        m = mths.index(l[1].strip())

        return '%s %2d %s %s %s' % (l[5], m, l[2], l[3], l[4])

    skipped = 0
    polled  = 0
    tweets  = []

    try:
        tweets = aT.GetPublicTimeline()
    except:
        dumpException('urllib2 error during public timeline poll')

    for tweet in tweets:
        polled += 1

        ts  = itemTimestamp(tweet.created_at)
        key = '%s_%s' % (ts, tweet.id)

        dItem = tweet.AsDict()
        dUser = tweet.user.AsDict()

        if sT.count(tweet.id) == 0:
            item = tf_store.Item()

            item.post_id        = dItem['id']
            item.post_timestamp = ts
            item.screen_name    = dUser['screen_name']
            item.name           = dUser['name']

            if 'text' in dItem:
                item.post = dItem['text']
            if 'location' in dUser:
                item.location = dUser['location']
            if 'profile_image_url' in dUser:
                item.imageurl = dUser['profile_image_url']
            if 'url' in dUser:
                item.url = dUser['url']
            if 'description' in dUser:
                item.desc = dUser['description']

            sT.update(item)
        else:
            skipped += 1

    print 'twitter timeline: polled %d, skipped %d' % (polled, skipped)


def pollPublicJaikuTimeline():
    """
    """
    skipped = 0
    polled  = 0
    posts   = []

    try:
        data = urllib2.urlopen('http://jaiku.com/feed/json').read()
    except:
        dumpException('urllib2 error during public timeline poll')

    print data
    posts = simplejson.loads(data)
    print posts

    print 'jaiku timeline: polled %d, skipped %d' % (polled, skipped)


def checkConfig():
    """
    Parse command-line for the config filename.
    Load config file and setup options.
    """
    cfg = { 'dbPath': '',
          }

    parser = OptionParser(version=version)

    parser.add_option('-c', '--config',  dest='configFile', default='tf.cfg')
    parser.add_option('-p', '--dbpath',  dest='dbPath',     default=cfg['dbPath'])

    (options, args) = parser.parse_args()
    options.args    = args

    if os.path.isfile(options.configFile):
        for item in open(options.configFile, 'r').readlines():
            if len(item) > 1:
                key, value = item[:-1].split('=', 1)
                key = key.strip()
                if key in cfg.keys():
                    setattr(options, key, value.strip())

    return options


def signalHandler(signum, frame):
    """
    Any signals received just dump out - nothing fancy
    """
    sys.stderr.write('Caught kill signal - terminating\n')
    sys.exit(signum)


if __name__ == "__main__":
    options = checkConfig()

    signal.signal(signal.SIGINT,  signalHandler)
    signal.signal(signal.SIGTERM, signalHandler)
    #signal.signal(signal.SIGHUP, signalHandler)

    global aT, sT

    aT = twitter.Api()
    sT = tf_store.Store(options.dbPath)

    while True:
        try:
            #pollPublicJaikuTimeline()
            pollPublicTwitterTimeline()
        except:
            dumpException('error during outer loop')

        time.sleep(60)

    sT.close()

